#!/bin/bash



####################################################
#   job_list 结构
# 1、ID 任务编号
# 2、DIR 任务文件夹
# 3、STATE 任务状态
# 4、NODE 计算节点
# 5、id 计算节点上的进程ID
####################################################
master_home=/home/cluster/master/
node_home=/home/cluster/node/
action_list=action_list.txt
job_list=job_list.txt

cat /dev/null > $master_home$action_list
cat /dev/null > $master_home$job_list

host_node=cluster@192.168.0.9
compute_node=( [0]=cluster@192.168.0.9 [1]=cluster@192.168.0.7 [2]=node3 )
declare -i NODE_NUM=2
declare -i Job_count=0

while [ 1 ]
do
#################轮询################################
  while [[ `head $master_home$action_list` ]]
  do
#################处理动作#############################
echo $'\n\n\n\n'
echo '==         add new job !         =='
    job=( `head -n 1 $master_home$action_list` )
      #################分配计算任务########################
    ##########查找空闲cpu ，返回节点及使用率#################
      declare -i num=0
      declare -i node_=0
      declare -i cpumin=100
      declare -i cpu=0
echo "############## cpu state ###############"
      while [[ $num -lt $NODE_NUM ]]
      do
	cpu=$(ssh ${compute_node[$num]} 'vmstat -n 1 2 | tail -1 | awk "{print \$(NF-3)+\$(NF-2)}"' )
echo "##   ${compute_node[$num]}  : $cpu  ##"
	if [ $cpu -lt $cpumin ]
	then
	  cpumin=$cpu
	  node_=$num
	fi
	num=$num+1
      done
echo "############## cpu state ###############"
      if [ $cpumin -lt 90 ]
      then
	Job_count=$Job_count+1
###修改主shell
	rm ${job[1]}/job_done.txt*
	cp ${job[1]}/main.sh ${job[1]}/$Job_count.sh
	echo "rm $Job_count.sh">> ${job[1]}/$Job_count.sh
	echo "rsync -avz -E $node_home$Job_count/ $host_node:${job[1]}">> ${job[1]}/$Job_count.sh
	echo "rm -R $node_home$Job_count">> ${job[1]}/$Job_count.sh
	rsync -avz -E ${job[1]}/ ${compute_node[$node_]}:$node_home$Job_count
        rm ${job[1]}/$Job_count.sh

	echo "${compute_node[$node_]} is doing job $Job_count"
	ssh ${compute_node[$node_]} "cd $node_home$Job_count && ./$Job_count.sh " &
	node_id=$(ssh ${compute_node[$node_]} "ps -A|grep $Job_count.sh|awk '{if(\$4==\"$Job_count.sh\") print \$1}'$Job_count.sh")
	node_id=( $node_id )
	echo "$Job_count   ${job[1]}   runing   ${compute_node[$node_]}   ${node_id[0]}" >>$master_home$job_list
	sed -i '1d' $master_home$action_list
      else
	echo "there is not enough cpu !"
	break
      fi

  done

################移除已完成任务#########################
cat  $master_home/$job_list | while read LINE
	do
	job=( $LINE )
    	if [ -f ${job[1]}/job_done.txt ]
    	then
      		sed -i "/^${job[0]}/"d $master_home$job_list
    	fi
	done

  sleep 10
done
